{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Density Estimation\n",
    "1. How to use normalizing flows for density estimation.\n",
    "2. Why you should use normalizing flows.\n",
    "3. Example: Anomaly detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.io import loadmat\n",
    "lympho = loadmat(\"lympho.mat\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10 | Train Loss 0.069\n",
      "Epoch 20 | Train Loss -4.492\n",
      "Epoch 30 | Train Loss -7.895\n",
      "Epoch 40 | Train Loss -9.233\n",
      "Epoch 50 | Train Loss -11.101\n",
      "Epoch 60 | Train Loss -13.374\n",
      "Epoch 70 | Train Loss -15.335\n",
      "Epoch 80 | Train Loss -15.898\n",
      "Epoch 90 | Train Loss -17.371\n",
      "Epoch 100 | Train Loss -17.292\n",
      "Epoch 110 | Train Loss -18.211\n",
      "Epoch 120 | Train Loss -18.836\n",
      "Epoch 130 | Train Loss -19.149\n",
      "Epoch 140 | Train Loss -18.746\n",
      "Epoch 150 | Train Loss -18.942\n",
      "Epoch 160 | Train Loss -19.928\n",
      "Epoch 170 | Train Loss -19.493\n",
      "Epoch 180 | Train Loss -18.708\n",
      "Epoch 190 | Train Loss -18.327\n",
      "Epoch 200 | Train Loss -20.113\n",
      "Epoch 210 | Train Loss -19.527\n",
      "Epoch 220 | Train Loss -19.837\n",
      "Epoch 230 | Train Loss -19.331\n",
      "Epoch 240 | Train Loss -19.364\n",
      "Epoch 250 | Train Loss -17.352\n",
      "Epoch 260 | Train Loss -18.578\n",
      "Epoch 270 | Train Loss -17.830\n",
      "Epoch 280 | Train Loss -16.697\n",
      "Epoch 290 | Train Loss -20.727\n",
      "Epoch 300 | Train Loss -19.390\n",
      "Epoch 310 | Train Loss -19.648\n",
      "Epoch 320 | Train Loss -19.635\n",
      "Epoch 330 | Train Loss -20.148\n",
      "Epoch 340 | Train Loss -17.182\n",
      "Epoch 350 | Train Loss -18.876\n",
      "Epoch 360 | Train Loss -17.321\n",
      "Epoch 370 | Train Loss -19.846\n",
      "Epoch 380 | Train Loss -18.767\n",
      "Epoch 390 | Train Loss -19.472\n",
      "Epoch 400 | Train Loss -20.134\n",
      "Epoch 410 | Train Loss -19.232\n",
      "Epoch 420 | Train Loss -19.583\n",
      "Epoch 430 | Train Loss -18.567\n",
      "Epoch 440 | Train Loss -20.473\n",
      "Epoch 450 | Train Loss -16.148\n",
      "Epoch 460 | Train Loss -18.441\n",
      "Epoch 470 | Train Loss -19.966\n",
      "Epoch 480 | Train Loss -19.316\n",
      "Epoch 490 | Train Loss -17.581\n",
      "Epoch 500 | Train Loss -19.473\n",
      "Epoch 510 | Train Loss -19.499\n",
      "Epoch 520 | Train Loss -17.684\n",
      "Epoch 530 | Train Loss -18.907\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-15d848a3646d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mEchoFlow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnb_blocks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblock_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"RNVP\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/Desktop/EchoFlow/echoflow/echoflow.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, df, context)\u001b[0m\n\u001b[1;32m    129\u001b[0m             \u001b[0mtrain_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mcontinuous\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcategorical\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontexts\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 131\u001b[0;31m                 \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    132\u001b[0m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/echoflow/lib/python3.8/site-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36mzero_grad\u001b[0;34m(self, set_to_none)\u001b[0m\n\u001b[1;32m    186\u001b[0m                         \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m                     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m                         \u001b[0;32mif\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    189\u001b[0m                             \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m                         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/echoflow/lib/python3.8/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mgrad\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    945\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mhandle_torch_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get__\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrelevant_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[attr-defined]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    946\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 947\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequires_grad\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"retains_grad\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_leaf\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_grad\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    948\u001b[0m             warnings.warn(\"The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad \"\n\u001b[1;32m    949\u001b[0m                           \u001b[0;34m\"attribute won't be populated during autograd.backward(). If you indeed want the gradient \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from echoflow import EchoFlow\n",
    "\n",
    "df = pd.DataFrame(lympho[\"X\"])\n",
    "\n",
    "model = EchoFlow(nb_blocks=8, block_type=\"RNVP\")\n",
    "model.fit(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: compute likelihoods, rank by likelihoods, show that outliers are lower likelihood"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}